# -*- coding: utf-8 -*-
import requests  # 导入requests
import random  # 取随机数
import time
from db.mysql_conn import MysqlUtils, config, sqlStr
mysql = MysqlUtils(config)
from config.conf import proxiesDEF, Referer_list
from config.useragent import user_agent_list
import re
get_goods_num = 0
goodsArr = []
user_agent_len = len(user_agent_list) - 10
Referer_list_len = len(Referer_list) - 1
def getAsin():
    goodsArr = mysql.query('select id, asin from best0711 where id < 173500 and id > 110000 and ranking=0 limit 500')
    if len(goodsArr) > 0:
        # proxies_list = proxiesDEF()
        proxies_list = ['218.24.16.198:58618', '47.106.197.184:8000', '106.75.8.141:808', '113.121.21.180:9999', '45.125.32.179:3128',
         '125.123.122.164:9000', '118.140.150.74:3128', '192.144.191.242:80', '221.122.91.60:80', '182.35.85.70:9999',
         '221.122.91.64:80', '182.35.81.3:9999', '183.146.239.69:8118', '171.13.102.15:9999', '123.163.122.160:9999',
         '58.56.149.198:53281', '113.140.1.82:53281', '122.143.50.208:80', '59.127.243.97:59324', '182.35.84.181:9999',
         '61.145.182.27:53281', '42.200.154.50:59175', '182.61.109.72:808', '27.42.168.46:54634', '123.206.77.34:8888',
         '114.220.15.177:61234', '175.16.20.48:80', '101.231.234.38:8080', '163.125.235.186:8118', '61.160.210.234:808',
         '175.42.123.101:9999', '36.249.48.15:9999', '123.163.97.209:9999', '182.35.86.67:9999', '1.197.10.218:9999',
         '182.35.83.3:9999', '163.204.246.101:9999', '124.205.143.213:61491', '123.146.97.184:53281',
         '203.110.164.139:52144', '116.252.39.176:53281', '221.7.255.168:8080', '111.75.223.9:30646',
         '121.226.188.115:808', '123.172.67.195:53281', '124.152.32.140:53281', '117.82.248.239:61234',
         '171.36.180.156:9797', '121.69.37.6:9797', '163.125.112.9:8118']

        # print(len(goodsArr))
        # print(proxies_list)
        getGoodsDeDetailTwo(goodsArr, proxies_list)
    else:
        print('==================爬完了================================')


def getGoodsDeDetailTwo (goodsArr, proxies_list):
    print('getGoodsDeDetailTwo')
    global get_goods_num
    for index, goods in enumerate(goodsArr):
        get_goods_num = get_goods_num + 1
        if get_goods_num % 400 == 0:
            print('休息一会==========马上开始')
            time.sleep(200 + random.uniform(0.1, 2.3))
        url = 'https://www.amazon.com/dp/' + goods['asin']
        try:
            proxies = {'http': proxies_list[index]}
        except:
            proxies = {'http': proxies_list[random.randint(0, len(proxies_list) - 1)]}
        headers = {
            'User-Agent': user_agent_list[random.randint(0, user_agent_len)],
            'Referer': Referer_list[random.randint(0, Referer_list_len)]
        }
        print(url)
        print(proxies)
        time.sleep(random.uniform(0.1, 1.0))
        try:
            session = requests.session()
            response = session.get(url, headers=headers, proxies=proxies, timeout=(3, 5))
            try:
                htmlstr = response.content.decode('utf-8')
                restr = re.compile('#(.*)\(<a href="/gp/bests')
                rankarr = restr.findall(htmlstr)[0].split(' in ')
                ranking = rankarr[0].replace(',', '')
                rankingtype = rankarr[1]
            except:
                ranking = -1
                rankingtype = ''
            if ranking == -1:
                try:
                    htmlstr = response.content.decode('utf-8')
                    restr = re.compile("<span>#(.*) \(<a href='/gp/b")
                    rankarr = restr.findall(htmlstr)[0].split(' in ')
                    ranking = rankarr[0].replace(',', '')
                    rankingtype = rankarr[1]
                except:
                    ranking = -1
                    rankingtype = ''
            if ranking == -1:
                try:
                    htmlstr = response.content.decode('utf-8')
                    restr = re.compile('#(.*)\(<a href="/gp/bests')
                    rankarr = restr.findall(htmlstr)[0].split(' in ')
                    ranking = rankarr[0].replace(',', '')
                    rankingtype = rankarr[1]
                except:
                    ranking = -1
                    rankingtype = ''


            if ranking == -1:
                print('=======没有采集到数据')
            print(str(goods['id']) + '   ' + goods['asin'] + '    ' + str(ranking) + '    ' + str(get_goods_num))
            d = {'ranking': ranking, 'rankingtype': rankingtype}  # 更新的字段及对应值
            t = ('id', goods['id'])  # 更新所需条件
            table_name = 'best0711'
            sqlStr = 'UPDATE %s SET ' % table_name + ','.join(['%s=%r' % (k, d[k]) for k in d]) + ',updateTime = NOW() WHERE %s=%r;' % (t[0], t[1])
            mysql.update(sqlStr)

            # goodsArr[index]['ranking'] = ranking
            # print(goods['asin'] + '    ' + str(goodsArr[index]['ranking']))
            # sqlParam = [goods['country'], goods['asin'], rankingtype, ranking, goods['star'], goods['reviews'],
            #             goods['moneytype'], goods['price'], goods['url'], goods['name'], goods['img']]
            # mysql.insert(sqlStr, sqlParam)
        except Exception as exc:
            print(exc)
            print('get 请求失败 跳往下一个')
    getAsin()



if __name__ == '__main__':
    getAsin()




